{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Advantage Actor-Critic to Play Acrobot-v1\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.distributions as distributions\n",
    "torch.manual_seed(0)\n",
    "\n",
    "imp.reload(logging)\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:23:09 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "22:23:09 [INFO] action_space: Discrete(3)\n",
      "22:23:09 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "22:23:09 [INFO] reward_range: (-inf, inf)\n",
      "22:23:09 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "22:23:09 [INFO] _max_episode_steps: 500\n",
      "22:23:09 [INFO] _elapsed_steps: None\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdvantageActorCriticAgent:\n",
    "    def __init__(self, env):\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.actor_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,],\n",
    "                output_size=env.action_space.n, output_activator=nn.Softmax(1))\n",
    "        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.0001)\n",
    "        self.critic_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                hidden_sizes=[100,])\n",
    "        self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.0002)\n",
    "        self.critic_loss = nn.MSELoss()\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        state_tensor = torch.as_tensor(observation, dtype=torch.float).reshape(1, -1)\n",
    "        prob_tensor = self.actor_net(state_tensor)\n",
    "        action_tensor = distributions.Categorical(prob_tensor).sample()\n",
    "        action = action_tensor.numpy()[0]\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, next_action \\\n",
    "                = self.trajectory[-8:]\n",
    "        state_tensor = torch.as_tensor(state, dtype=torch.float).unsqueeze(0)\n",
    "        next_state_tensor = torch.as_tensor(next_state, dtype=torch.float).unsqueeze(0)\n",
    "\n",
    "        # calculate TD error\n",
    "        next_v_tensor = self.critic_net(next_state_tensor)\n",
    "        target_tensor = reward + (1. - done) * self.gamma * next_v_tensor\n",
    "        v_tensor = self.critic_net(state_tensor)\n",
    "        td_error_tensor = target_tensor - v_tensor\n",
    "        \n",
    "        # train actor\n",
    "        pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "        logpi_tensor = torch.log(pi_tensor.clamp(1e-6, 1.))\n",
    "        actor_loss_tensor = -(self.discount * td_error_tensor * logpi_tensor).squeeze()\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss_tensor.backward(retain_graph=True)\n",
    "        self.actor_optimizer.step()\n",
    "        \n",
    "        # train critic\n",
    "        pred_tensor = self.critic_net(state_tensor)\n",
    "        critic_loss_tensor = self.critic_loss(pred_tensor, target_tensor)\n",
    "        self.critic_optimizer.zero_grad()\n",
    "        critic_loss_tensor.backward()\n",
    "        self.critic_optimizer.step()\n",
    "\n",
    "\n",
    "agent = AdvantageActorCriticAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:23:09 [INFO] ==== train ====\n",
      "22:23:11 [DEBUG] train episode 0: reward = -500.00, steps = 500\n",
      "22:23:13 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "22:23:15 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "22:23:18 [DEBUG] train episode 3: reward = -500.00, steps = 500\n",
      "22:23:20 [DEBUG] train episode 4: reward = -500.00, steps = 500\n",
      "22:23:22 [DEBUG] train episode 5: reward = -500.00, steps = 500\n",
      "22:23:24 [DEBUG] train episode 6: reward = -500.00, steps = 500\n",
      "22:23:26 [DEBUG] train episode 7: reward = -500.00, steps = 500\n",
      "22:23:28 [DEBUG] train episode 8: reward = -500.00, steps = 500\n",
      "22:23:30 [DEBUG] train episode 9: reward = -500.00, steps = 500\n",
      "22:23:33 [DEBUG] train episode 10: reward = -500.00, steps = 500\n",
      "22:23:35 [DEBUG] train episode 11: reward = -500.00, steps = 500\n",
      "22:23:37 [DEBUG] train episode 12: reward = -500.00, steps = 500\n",
      "22:23:39 [DEBUG] train episode 13: reward = -500.00, steps = 500\n",
      "22:23:41 [DEBUG] train episode 14: reward = -500.00, steps = 500\n",
      "22:23:44 [DEBUG] train episode 15: reward = -500.00, steps = 500\n",
      "22:23:46 [DEBUG] train episode 16: reward = -500.00, steps = 500\n",
      "22:23:48 [DEBUG] train episode 17: reward = -500.00, steps = 500\n",
      "22:23:50 [DEBUG] train episode 18: reward = -500.00, steps = 500\n",
      "22:23:52 [DEBUG] train episode 19: reward = -500.00, steps = 500\n",
      "22:23:55 [DEBUG] train episode 20: reward = -500.00, steps = 500\n",
      "22:23:57 [DEBUG] train episode 21: reward = -500.00, steps = 500\n",
      "22:23:59 [DEBUG] train episode 22: reward = -500.00, steps = 500\n",
      "22:24:01 [DEBUG] train episode 23: reward = -500.00, steps = 500\n",
      "22:24:03 [DEBUG] train episode 24: reward = -500.00, steps = 500\n",
      "22:24:05 [DEBUG] train episode 25: reward = -500.00, steps = 500\n",
      "22:24:08 [DEBUG] train episode 26: reward = -500.00, steps = 500\n",
      "22:24:10 [DEBUG] train episode 27: reward = -500.00, steps = 500\n",
      "22:24:12 [DEBUG] train episode 28: reward = -500.00, steps = 500\n",
      "22:24:14 [DEBUG] train episode 29: reward = -500.00, steps = 500\n",
      "22:24:16 [DEBUG] train episode 30: reward = -500.00, steps = 500\n",
      "22:24:19 [DEBUG] train episode 31: reward = -500.00, steps = 500\n",
      "22:24:20 [DEBUG] train episode 32: reward = -457.00, steps = 458\n",
      "22:24:23 [DEBUG] train episode 33: reward = -500.00, steps = 500\n",
      "22:24:25 [DEBUG] train episode 34: reward = -500.00, steps = 500\n",
      "22:24:26 [DEBUG] train episode 35: reward = -357.00, steps = 358\n",
      "22:24:28 [DEBUG] train episode 36: reward = -500.00, steps = 500\n",
      "22:24:30 [DEBUG] train episode 37: reward = -291.00, steps = 292\n",
      "22:24:31 [DEBUG] train episode 38: reward = -387.00, steps = 388\n",
      "22:24:33 [DEBUG] train episode 39: reward = -299.00, steps = 300\n",
      "22:24:33 [DEBUG] train episode 40: reward = -215.00, steps = 216\n",
      "22:24:35 [DEBUG] train episode 41: reward = -269.00, steps = 270\n",
      "22:24:36 [DEBUG] train episode 42: reward = -238.00, steps = 239\n",
      "22:24:36 [DEBUG] train episode 43: reward = -182.00, steps = 183\n",
      "22:24:37 [DEBUG] train episode 44: reward = -152.00, steps = 153\n",
      "22:24:38 [DEBUG] train episode 45: reward = -148.00, steps = 149\n",
      "22:24:38 [DEBUG] train episode 46: reward = -148.00, steps = 149\n",
      "22:24:39 [DEBUG] train episode 47: reward = -148.00, steps = 149\n",
      "22:24:40 [DEBUG] train episode 48: reward = -171.00, steps = 172\n",
      "22:24:40 [DEBUG] train episode 49: reward = -169.00, steps = 170\n",
      "22:24:41 [DEBUG] train episode 50: reward = -150.00, steps = 151\n",
      "22:24:42 [DEBUG] train episode 51: reward = -134.00, steps = 135\n",
      "22:24:42 [DEBUG] train episode 52: reward = -103.00, steps = 104\n",
      "22:24:43 [DEBUG] train episode 53: reward = -144.00, steps = 145\n",
      "22:24:44 [DEBUG] train episode 54: reward = -240.00, steps = 241\n",
      "22:24:45 [DEBUG] train episode 55: reward = -215.00, steps = 216\n",
      "22:24:46 [DEBUG] train episode 56: reward = -152.00, steps = 153\n",
      "22:24:46 [DEBUG] train episode 57: reward = -223.00, steps = 224\n",
      "22:24:47 [DEBUG] train episode 58: reward = -211.00, steps = 212\n",
      "22:24:48 [DEBUG] train episode 59: reward = -94.00, steps = 95\n",
      "22:24:48 [DEBUG] train episode 60: reward = -147.00, steps = 148\n",
      "22:24:49 [DEBUG] train episode 61: reward = -142.00, steps = 143\n",
      "22:24:50 [DEBUG] train episode 62: reward = -169.00, steps = 170\n",
      "22:24:50 [DEBUG] train episode 63: reward = -120.00, steps = 121\n",
      "22:24:51 [DEBUG] train episode 64: reward = -249.00, steps = 250\n",
      "22:24:52 [DEBUG] train episode 65: reward = -142.00, steps = 143\n",
      "22:24:52 [DEBUG] train episode 66: reward = -105.00, steps = 106\n",
      "22:24:53 [DEBUG] train episode 67: reward = -138.00, steps = 139\n",
      "22:24:54 [DEBUG] train episode 68: reward = -134.00, steps = 135\n",
      "22:24:54 [DEBUG] train episode 69: reward = -105.00, steps = 106\n",
      "22:24:55 [DEBUG] train episode 70: reward = -132.00, steps = 133\n",
      "22:24:55 [DEBUG] train episode 71: reward = -131.00, steps = 132\n",
      "22:24:56 [DEBUG] train episode 72: reward = -130.00, steps = 131\n",
      "22:24:57 [DEBUG] train episode 73: reward = -156.00, steps = 157\n",
      "22:24:57 [DEBUG] train episode 74: reward = -142.00, steps = 143\n",
      "22:24:58 [DEBUG] train episode 75: reward = -113.00, steps = 114\n",
      "22:24:58 [DEBUG] train episode 76: reward = -136.00, steps = 137\n",
      "22:24:59 [DEBUG] train episode 77: reward = -140.00, steps = 141\n",
      "22:25:00 [DEBUG] train episode 78: reward = -177.00, steps = 178\n",
      "22:25:01 [DEBUG] train episode 79: reward = -187.00, steps = 188\n",
      "22:25:01 [DEBUG] train episode 80: reward = -124.00, steps = 125\n",
      "22:25:02 [DEBUG] train episode 81: reward = -123.00, steps = 124\n",
      "22:25:02 [DEBUG] train episode 82: reward = -139.00, steps = 140\n",
      "22:25:03 [DEBUG] train episode 83: reward = -112.00, steps = 113\n",
      "22:25:03 [DEBUG] train episode 84: reward = -113.00, steps = 114\n",
      "22:25:04 [DEBUG] train episode 85: reward = -112.00, steps = 113\n",
      "22:25:04 [DEBUG] train episode 86: reward = -87.00, steps = 88\n",
      "22:25:04 [DEBUG] train episode 87: reward = -93.00, steps = 94\n",
      "22:25:05 [DEBUG] train episode 88: reward = -122.00, steps = 123\n",
      "22:25:06 [DEBUG] train episode 89: reward = -115.00, steps = 116\n",
      "22:25:06 [INFO] ==== test ====\n",
      "22:25:06 [DEBUG] test episode 0: reward = -115.00, steps = 116\n",
      "22:25:06 [DEBUG] test episode 1: reward = -193.00, steps = 194\n",
      "22:25:06 [DEBUG] test episode 2: reward = -128.00, steps = 129\n",
      "22:25:06 [DEBUG] test episode 3: reward = -119.00, steps = 120\n",
      "22:25:06 [DEBUG] test episode 4: reward = -96.00, steps = 97\n",
      "22:25:06 [DEBUG] test episode 5: reward = -120.00, steps = 121\n",
      "22:25:06 [DEBUG] test episode 6: reward = -117.00, steps = 118\n",
      "22:25:06 [DEBUG] test episode 7: reward = -144.00, steps = 145\n",
      "22:25:06 [DEBUG] test episode 8: reward = -125.00, steps = 126\n",
      "22:25:07 [DEBUG] test episode 9: reward = -124.00, steps = 125\n",
      "22:25:07 [DEBUG] test episode 10: reward = -124.00, steps = 125\n",
      "22:25:07 [DEBUG] test episode 11: reward = -129.00, steps = 130\n",
      "22:25:07 [DEBUG] test episode 12: reward = -105.00, steps = 106\n",
      "22:25:07 [DEBUG] test episode 13: reward = -112.00, steps = 113\n",
      "22:25:07 [DEBUG] test episode 14: reward = -144.00, steps = 145\n",
      "22:25:07 [DEBUG] test episode 15: reward = -124.00, steps = 125\n",
      "22:25:07 [DEBUG] test episode 16: reward = -124.00, steps = 125\n",
      "22:25:07 [DEBUG] test episode 17: reward = -94.00, steps = 95\n",
      "22:25:07 [DEBUG] test episode 18: reward = -110.00, steps = 111\n",
      "22:25:07 [DEBUG] test episode 19: reward = -103.00, steps = 104\n",
      "22:25:08 [DEBUG] test episode 20: reward = -119.00, steps = 120\n",
      "22:25:08 [DEBUG] test episode 21: reward = -86.00, steps = 87\n",
      "22:25:08 [DEBUG] test episode 22: reward = -124.00, steps = 125\n",
      "22:25:08 [DEBUG] test episode 23: reward = -129.00, steps = 130\n",
      "22:25:08 [DEBUG] test episode 24: reward = -112.00, steps = 113\n",
      "22:25:08 [DEBUG] test episode 25: reward = -118.00, steps = 119\n",
      "22:25:08 [DEBUG] test episode 26: reward = -143.00, steps = 144\n",
      "22:25:08 [DEBUG] test episode 27: reward = -109.00, steps = 110\n",
      "22:25:08 [DEBUG] test episode 28: reward = -97.00, steps = 98\n",
      "22:25:08 [DEBUG] test episode 29: reward = -189.00, steps = 190\n",
      "22:25:09 [DEBUG] test episode 30: reward = -115.00, steps = 116\n",
      "22:25:09 [DEBUG] test episode 31: reward = -170.00, steps = 171\n",
      "22:25:09 [DEBUG] test episode 32: reward = -111.00, steps = 112\n",
      "22:25:09 [DEBUG] test episode 33: reward = -110.00, steps = 111\n",
      "22:25:09 [DEBUG] test episode 34: reward = -103.00, steps = 104\n",
      "22:25:09 [DEBUG] test episode 35: reward = -121.00, steps = 122\n",
      "22:25:09 [DEBUG] test episode 36: reward = -133.00, steps = 134\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:25:09 [DEBUG] test episode 37: reward = -132.00, steps = 133\n",
      "22:25:09 [DEBUG] test episode 38: reward = -136.00, steps = 137\n",
      "22:25:09 [DEBUG] test episode 39: reward = -122.00, steps = 123\n",
      "22:25:10 [DEBUG] test episode 40: reward = -125.00, steps = 126\n",
      "22:25:10 [DEBUG] test episode 41: reward = -130.00, steps = 131\n",
      "22:25:10 [DEBUG] test episode 42: reward = -118.00, steps = 119\n",
      "22:25:10 [DEBUG] test episode 43: reward = -103.00, steps = 104\n",
      "22:25:10 [DEBUG] test episode 44: reward = -99.00, steps = 100\n",
      "22:25:10 [DEBUG] test episode 45: reward = -120.00, steps = 121\n",
      "22:25:10 [DEBUG] test episode 46: reward = -119.00, steps = 120\n",
      "22:25:10 [DEBUG] test episode 47: reward = -140.00, steps = 141\n",
      "22:25:10 [DEBUG] test episode 48: reward = -98.00, steps = 99\n",
      "22:25:10 [DEBUG] test episode 49: reward = -97.00, steps = 98\n",
      "22:25:10 [DEBUG] test episode 50: reward = -128.00, steps = 129\n",
      "22:25:10 [DEBUG] test episode 51: reward = -136.00, steps = 137\n",
      "22:25:11 [DEBUG] test episode 52: reward = -94.00, steps = 95\n",
      "22:25:11 [DEBUG] test episode 53: reward = -186.00, steps = 187\n",
      "22:25:11 [DEBUG] test episode 54: reward = -108.00, steps = 109\n",
      "22:25:11 [DEBUG] test episode 55: reward = -129.00, steps = 130\n",
      "22:25:11 [DEBUG] test episode 56: reward = -92.00, steps = 93\n",
      "22:25:11 [DEBUG] test episode 57: reward = -112.00, steps = 113\n",
      "22:25:11 [DEBUG] test episode 58: reward = -104.00, steps = 105\n",
      "22:25:11 [DEBUG] test episode 59: reward = -107.00, steps = 108\n",
      "22:25:11 [DEBUG] test episode 60: reward = -142.00, steps = 143\n",
      "22:25:11 [DEBUG] test episode 61: reward = -162.00, steps = 163\n",
      "22:25:11 [DEBUG] test episode 62: reward = -105.00, steps = 106\n",
      "22:25:12 [DEBUG] test episode 63: reward = -111.00, steps = 112\n",
      "22:25:12 [DEBUG] test episode 64: reward = -118.00, steps = 119\n",
      "22:25:12 [DEBUG] test episode 65: reward = -129.00, steps = 130\n",
      "22:25:12 [DEBUG] test episode 66: reward = -91.00, steps = 92\n",
      "22:25:12 [DEBUG] test episode 67: reward = -122.00, steps = 123\n",
      "22:25:12 [DEBUG] test episode 68: reward = -142.00, steps = 143\n",
      "22:25:12 [DEBUG] test episode 69: reward = -138.00, steps = 139\n",
      "22:25:12 [DEBUG] test episode 70: reward = -105.00, steps = 106\n",
      "22:25:12 [DEBUG] test episode 71: reward = -165.00, steps = 166\n",
      "22:25:12 [DEBUG] test episode 72: reward = -144.00, steps = 145\n",
      "22:25:13 [DEBUG] test episode 73: reward = -176.00, steps = 177\n",
      "22:25:13 [DEBUG] test episode 74: reward = -143.00, steps = 144\n",
      "22:25:13 [DEBUG] test episode 75: reward = -144.00, steps = 145\n",
      "22:25:13 [DEBUG] test episode 76: reward = -135.00, steps = 136\n",
      "22:25:13 [DEBUG] test episode 77: reward = -132.00, steps = 133\n",
      "22:25:13 [DEBUG] test episode 78: reward = -103.00, steps = 104\n",
      "22:25:13 [DEBUG] test episode 79: reward = -125.00, steps = 126\n",
      "22:25:13 [DEBUG] test episode 80: reward = -84.00, steps = 85\n",
      "22:25:13 [DEBUG] test episode 81: reward = -123.00, steps = 124\n",
      "22:25:13 [DEBUG] test episode 82: reward = -100.00, steps = 101\n",
      "22:25:13 [DEBUG] test episode 83: reward = -127.00, steps = 128\n",
      "22:25:14 [DEBUG] test episode 84: reward = -87.00, steps = 88\n",
      "22:25:14 [DEBUG] test episode 85: reward = -193.00, steps = 194\n",
      "22:25:14 [DEBUG] test episode 86: reward = -103.00, steps = 104\n",
      "22:25:14 [DEBUG] test episode 87: reward = -120.00, steps = 121\n",
      "22:25:14 [DEBUG] test episode 88: reward = -94.00, steps = 95\n",
      "22:25:14 [DEBUG] test episode 89: reward = -117.00, steps = 118\n",
      "22:25:14 [DEBUG] test episode 90: reward = -123.00, steps = 124\n",
      "22:25:14 [DEBUG] test episode 91: reward = -94.00, steps = 95\n",
      "22:25:14 [DEBUG] test episode 92: reward = -117.00, steps = 118\n",
      "22:25:14 [DEBUG] test episode 93: reward = -126.00, steps = 127\n",
      "22:25:14 [DEBUG] test episode 94: reward = -167.00, steps = 168\n",
      "22:25:15 [DEBUG] test episode 95: reward = -121.00, steps = 122\n",
      "22:25:15 [DEBUG] test episode 96: reward = -90.00, steps = 91\n",
      "22:25:15 [DEBUG] test episode 97: reward = -93.00, steps = 94\n",
      "22:25:15 [DEBUG] test episode 98: reward = -141.00, steps = 142\n",
      "22:25:15 [DEBUG] test episode 99: reward = -103.00, steps = 104\n",
      "22:25:15 [INFO] average episode reward = -122.29 ± 23.37\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXjcd33g8fdnbo1Gt2TZ1uEjlp3YTuIkjhMSCCHkLC0Blrahuxtol01Lw/YCWlj6bNvtk73oQXm20CeFLc22BVKgmxQaIKEJSUhiR87lI47t2LItS7bua6S5v/vH7/cbjTSjy6Njjs/refRI+s1o9PPPms985/P9fD9fMcaglFKqvLjW+gSUUkqtPg3+SilVhjT4K6VUGdLgr5RSZUiDv1JKlSHPWp/AYjU2NprNmzev9WkopVRROXjw4IAxpmn28aIJ/ps3b6azs3OtT0MppYqKiJzJdVzTPkopVYY0+CulVBnS4K+UUmVIg79SSpUhDf5KKVWGNPgrpVQZ0uCvlFJlSIO/Ukqtsje6R/jpyYE1PQcN/koptco+9ejr/Nuv7ueLTx1nrfZU0eCvlFKrqHt4khN9E2xqCPLFp07wG998jUg8uernocFfKUU4mlizEWi5eeatfgC+9tHr+b27L+d7b/Rw38MvrfoLQF7BX0R+XkSOiEhKRPbOuu1zInJSRN4Skbsyjl8nIofs274kIpLPOSil8jMVS3Ljf/sx/++182t9KmXhmbf6aa2r4LKmSj5x62V84cNX89q5EZ493r+q55HvyP8w8CHg2cyDIrITuA/YBdwNfFlE3PbNXwEeADrsj7vzPAelVB4GJqKMRxO8fm50rU+l5EUTSV54e4BbdzThjHt/7uoN+D0uXjo1tKrnklfwN8a8aYx5K8dN9wLfNMZEjTGngZPAPhHZAFQbY1401nvMR4AP5HMOSqn8jEcSAJwbmlzjMyl9nV3DTMaS3Lp9XfqY3+PmmvZa9p8eXNVzWamcfwtwLuP7bvtYi/317OM5icgDItIpIp39/av7lkipcjEWiQNwRoP/inv6WB8+t4ubtjXMOH7DlgaO9o4xOhVftXNZMPiLyFMicjjHx73z/ViOY2ae4zkZYx42xuw1xuxtasrai0CpspBKGR78h1dWLCecOfJPpXTSdyU9c7yfG7bWE/TN3Erlhq31GAOdXauX+lkw+BtjbjfG7M7x8dg8P9YNtGV83wr02MdbcxxXSs3h1MAE33+jl++9sTJPlTF7tBlNpOgbjy77409EE/zCX73Ioe6F5xQ++503eOTFrmU/h0LQPTzJyb4J3r09eyB7bXsdPreL/acLKPhfoseB+0TELyJbsCZ2DxhjeoFxEbnRrvK5H5jvRUSpstfZNQzAWxfGV+TxnbQPwNkVSP28dnaEA11DfP9Q77z3G52K863Oc/zdSzk3nip6TonnrTvWZd0W8Lq5uq2G/adWL++fb6nnB0WkG3gH8H0R+SGAMeYI8ChwFPgB8KAxxili/QTwVaxJ4LeBJ/I5B6VKXecZK/gfvzixImkZJ+0DcGYwvOyPf6THGvG/enZ43vt1dg1hjPXv7BuLLPt5rLXMEs9cbtjSwOGeMSaiiZy3L7e89vA1xvwT8E9z3PYQ8FCO453A7nx+r1Ll5OCZYTwuYSqe5NzwJJsacgePSzU2FcfncZFIplak4udwzxgAh86PkkwZ3K7cS3syUx4/fXuAD17TmvN+hezRznOMTsbZ1BBkc2MlQZ+bs4OTnBma5IW3B/jQtS3MtbTphq31/O+nT9LZNZTz3cFyK5oN3JUqRwMTUU4PhLln93qeOHyBYxfGlz/4R+LUBb143a4Vqfg50jOKz+NiMpbk+MVxrthQnfN++08NsndTHSf7J/jpycFlCf4vnBzggf97kB/+9i201Fbk/Xjz6RuP8LvffmPO2wNeF++/es7iRq7bVIfHJew/PR38kynD2/0TbG+uWvbz1eCvVAE7aKd8PrKvnScOX+D4hXHu2rV+WX/HeCRBdcBLU5V/2XP+4WiC0wNhPrinhe++ep5Xz47kDP4T0QSHe8b49Vsvo7k6wE9PDmCMmXOUvFg/PHKBiWiCJw718vF3bc3rsRbywkkrX/8PH7+BSr+HrsEwk7Ekm+qDtDcE2VBTMee7HoCgz8OVrdN5/2TK8Jl/fJ0nDl/gyd+5hda64LKer/b2UaqAdXYN4fO4uGFrPW31FRy7uPyTvmOROFUBD5sagpwdXN7g/2bvGMbAPVduoC7o5bVzufP+nV1DJFOGG7Y0cPO2RnpHI5wayH/+wUkl/ejoxbwfayHPnxygNujlxq0NXN1Wy717WvjIvnZu2tZIa11w3sDvuGFLA290jzIWifNb33qN7756nl+/9bJlD/ygwV+pgtZ5ZpirWmrwe9zsaK7i+ApU/IxHElRXeGmrDzIYji16wvHc0CR/+PgREsnUnPc5Yuf7r2ypYU9bLa+dG8l5v/2nh/C4hGs31XKzvQAq3373w+EYxy6MUxv00tk1xFA4ltfjzccYw09PDnDzZY24FhHk53LD1noSKcMv/NWL/PPrPXz2nsv5T+/tWMYznabBX6kCFYknOXx+lOs21wGwY30VpwbCRBPL2/1xbCpOdcDLpnprLmGxo//vH+rl6y90cW54as77HD4/SkOlj+ZqP3va6jjRN8F4JHsV6/5Tg1zVWkPQ56G9PkhrXUXewf+AvWDqN27rIGXgx2/mN/o3xjAwEeXgmeGsF5JTA2F6RyPcvK0xr9+xd1MdLoFjF8b5/fddwa+9+7K8Hm8+mvNXqkC90T1KPGnYu6kegB3rq0mmDKf6w3NOml6KsUiCqoAVdAHODoXZuXHhx++y0zITkbnfKRzpGWPnxmpEhGvaazHG+ndlBsnJWII3ukf5j7dYOXkR4Z3bGvn+od55q4MWsv/UEH6Pi1+6oZ2/fu4UTx69yM/vbVv4B2fpG4vwq393kBMXJ9LvivZtqefRX31H+j4v2C9UN89q27BUVQEvn7pzB83VAT583cpWO+nIX6kC1XnGGrlet8ke+dsVH8u52MsYw3gkTnWFl/YGJ/gvbuTv5OTHo7n70UQTVnXP7pYaAK5uqwWy6/1fOTNCImW4YUt9+tjN2xoZjyQ4dP7SO43uPz3INe21BLxu7tjZzLMn+pmKLf1d0ytnR3j17Ah37mzmD35uJ/e/YxMHTg/xekYK6/mTA7TWVaRfQPPx4Hu2rXjgBw3+ShWsg13DbG2qpL7SB8DWpkq8buHYMgb/SDxFPGmoDnipqbA+ziwy7bPQyP/ExQkSKcMu+11ETYWXy5oqs/L++08P4nYJezdPB/+bLssv7z86Fedo7xg3brUe586d64nEUzx/CY/XO2qltT7/viv45Zu38Jm7dhDye/ja86cBqyrnhbcHeee2xryrk1aTpn2UKkCplOHg2WHu3NmcPuZ1u7isKcTxZaz4cVo7VAWsULCpIbiokX84mkj3AZprgviwPWrfvbEmfWxPWx0/Od43o4xz/6khdm+sJuSfDkcNIT87N1Tz7PF+7tm9njNDk5wZCNM1OMmZwTBnhibZ1hTi4fv3kouzWviGLVbwv2FrPVUBDz86coE7Mq7pYvSORvB7XOkX4aqAl/uub+NvXujis/dcTt94lPFIIu98/2rT4K9UATo1MMHIZDyd73dsb65K1/4vB2fytbrCC0BbfTAdtOdzOqMMc3yOkf+RnjFCfs+MVMg17bV855VuuoenaKsPEoknee3cCB+7eXPWz9+8rYG/fu40t/3pT9LHgj43mxoqCXjc/OjoRXpHp9hQk714a//pIXxuF9e0W6kmr9vFbZev48fH+pY8j9AzMsWGmsCMUf3Hbt7M37zQxd++0JW+ds67lWKhwV+pAvTC29ZCH6fSx7FjfRWPv97DeCROVcCb9+8ZnbICd7Uz8q8P8sPDF0gkU3jcc2eFuzJ6AM058u8ZZeeG6hmlj3ucvP+5ESp8bv7XD44RS6Zm5Psd/+GdW6mp8LK+poLNDdZCqaaQHxHh2IUx7v7iczx3fIBfuD57Enf/qUH2tFn5fscdO5t57LUeDp4ZZl+O3zeX3tFI1gtMa12Qe3av5x8OnGVrU4grNlTTEPIv+jELgeb8lSowxhgeefEMOzdUs7VxZisHZ9J3uVI/4+m0j/VC0l4fJJEy9I7O31jNyfe7XZJz5J9MGd7sHWNXy8yqocvXVxHwuvirZ97mPV94hu+8cp6P3bQ5Zy+b9TUBPnlbBx++rpW9m+tZVzU9+t7RXMW6Kj/Pnsje48BZLXzD1pkB/t3bm/C5XfzB40eW1De/d2SKDbWBrOMff9dWxiMJXj83wjvzrPJZCxr8lSowPznez8m+Cf7jLVuyJhB3rLeC/3JN+o7Zgbumwhr5L7bi59RAmPXVAWoqvEzkqPY51T9BJJ5iV0a+H8DjdnF1ay1He8fYt6WeH/7Wu/jD9+9acjmniPCujiaePzlAclan08zVwpmqAl7+4r49DIWjfPivXuTX//7ggmsaEskUF8YibMyRWtrTVsv19juzYsv3gwZ/pQrO154/TXO1n/dduTHrtta6Cip97mVb6ets5JI58gdmVPwM51gZ2zUQZnNjkJDfk7Pax1nZu7sle73AFz58Nd/99Zv42seuZ9u6S29Ydsv2RkYm41lzFJmrhWe758oNPP3pW/nt27fz9LF+PvSVnxKfZ4Vy33iUlCHnyB/gU3fu4IYt9VkvNMVAg79SBeTN3jGeOzHAR2/ajM+T/fQUEbavr1q2kb+Tsqm2g/+Gmgq8buHs0CRjkTi/863XuOaPn+SlWZuMdA1OsqUxZAX/HDn/YxfG8bqFy5pCWbe1NwS5tr0u6/hSvdMebT+XkfpJpQxPHb3I1W21WVslOoI+D795ewf/7UO7GZiIcbJvYs7f4ZR55hr5A9y4tYFv/eo7qPC5c95eyDT4K1VAvvb8aSq8bv7tvk1z3mf7uire7p87YC3FWCSO1y0EvFYocLuE1rqgVWL5xed47PUe3C7hqYzGaKOTcYbCMbY0BqkKeNKpo0xD4Sh1QR/eeSaN89UQ8rO7pZpnj0/X7v/L4V5O9E1w/zvmvn6OK1usdwbOu5RcekasuY+5Rv7FTKt9lFoGh7pH06PE2bxuFzdta8DvmX902DcW4bHXzvNL+9qpCc5dydNU5WcoHCOVMnk1EQPSVUOZcwvt9UF+cryfzQ1Bvv1r7+ALP3xrxuKo03alz5bGEFWBoXSAzDQ6FaemIv9qpIW8q6OJv372FOOROEGfhy8+dYKOdSF+9qrslNlsWxorqfC6OXx+dM4Vtc7/aa5y0mKnwV+pPLzdP8F//5djPLVA07Dff98VC/aTf+TFMyRShl++ecu896sNekkZa9ReG/Qt+ZwzjU0l0mWejo/dvJkrW2r4xK2XUen3cPO2Rr7ww7cYmIjSGPKnK322ODn/HGmf0ak4tfO8gC2Xd3U08pVn3ubFtweZjCU52TfBX/7StYuaQHa7hCs2VHF0gZF/pc+ddY1KQen9i5RaBZF4kv/xxDH+7qUzBLxuPnPXDt69vSnnfT/9j6/zg8MX5g3+8WSKbxw4y+1XNLO5cf6duursgD88OX/wTyRTvHhqkHd15D4vsF5AqmeN0N+zYx3vySi9dIL/C28P8v6rN3JqIIxLrAVhocBcwT9ByyqkSq7bVEfQ5+bpt/p56dQgl6+v4p7di9/sZndLDd852D3nu6je0Sk21FYUVduGxdKcv1KX4B8PdvP1F7r4hevbeOYzt/Lge7axu6Um58ddu9Zz8OwwAxPROR/v6WN9DIZjfGTfwl0n6yqtYD08OX9/+iePXuTff+0Axy7MPbIdtzt6zufKlhqqAh5+esJK/XQNhGmpq8DvcRPye3NW+4xNZb+orAS/x82NWxt4tPMcpwfC/M4d25eUCtu1sZpwLDnn9pXWAq/Sy/eDBn+lLsn+U4NsqAnw0Ad207jAys47dzVjFugn/+2D3TSG/Nwyzyjd4Yz2RxYI/k5AOz9Pv32nl/983C7hpssaeN7eWrFrMMxmex/hqoCHWDKVtcfAauX8wUr9JFOGK1tqlty3x1mHMFdLi56R3DX+pUCDv1JLZIzh5a4hrt9cv6h0wM4N1bTUVvDkHFsJDkxE+ddjfXzo2pZ5Wyo40mmfcO5Wyo7eESvoXxyb+x3HWGTh4A9WWeX5kSnODE5yuj/Mlsbp4A8z+/skkikmoolVC/63X9FMY8jHZ++5fMnpme3NVXjdkrPiJ5pIMjARLclKH9Ccv1JLdm5oiotjUa5fZH8YEeGOnc1848BZJmOJrPrzx17rIZEyi+7hXhdcXNqnx27RcHFs7lYNi0n7ANxk19T/8+s9jEcT6eDvdOKciCTS74CmVw2vTvBvqw/S+ft3XNLP+jwutjdXcaQne+R/cdR60dSRv1IKmN4ecN/mxTcHu3NnM9FEakZNuuPbB7u5urWG7c2LW+1aHfDiEhiZXGDkb5cpOq2XZ4snU0zGkovKzW9trGRDTYBvHDgLkJ6UTgf/jEnfUXvV8GoF/3zt2ljNkZ4xjJnZJqLHKfMs0ZG/Bn+llujl00PUVHjpWJe9enUu12+pp6bCy4+OXphx/EjPKG/2ji1p5yaXS6ip8C448u+16+/75hj5OxO1ixn5iwg3b2tMv5vYYuf8QznSPs5cRPEE/xqGwjEuzLpOpVzjDxr8lVqyl7uGrI22l1BV4vST/9djfSQyesn8Y2c3PreLn7t64UVJmeqCvnlH/pF4kkG7J8/F8dzB39nIZTE5f5jen9bjElrrrIBY5bd+tphH/k7/ocPnZ+b9ncVrG3Xkr5TqH49yaiC86Hx/pjt3NjMyGeflLmszlologsdeO88du5qXvFirNjj/yP+CPUL3e1xzTviOOb38Fxmkb77Myvu31wfTE9POu4bMzp7FFvwvX1+NCFl5/97RKWqD3jl7BBW70vxXKbVCnD7w1y8h3++4ZXsTPo+LP3/qOH/5tIsDp4eIJVPcl2MzkoXUV/o4n6OtgsPJV1/ZUsMrZ4dzbs4yPmsLx4Wsqw5wZUtNuu0z5E77OJ1C52tRUUgq/R62NlZmjfx7R7I3cSklGvyVWoIDXUMEvC6ubKlZ+M6zVPo93LZjHT84coHtzSE+etMmbr+imRu2Lr0dcG3QN29DMiffv6etls4zwwyGYzRXz0xfLDXtA/DIr+zD7Z5OdzkTvpnBv9hG/mDl/Wdv8NIzGmFjiS7wAg3+Si3Jy11D7GmrzdlueTH+7Bev5r9GdrGuOr+gUrdA2seZrLzK3jbx4lgkO/in0z6LDwN1lTPTU36PC69bsnL+Aa9rwUZ2hWTXxmoef72HoXAsvVF77+gU1+XYE6BUaM5fqUUaj8Q52jO2pBLP2YI+T96BH6yRfySeIhJP5ry9ZzRCfaWPTfbmLLny/mOztnC8FCKStaHLaq7uXS677XdyTt5/KpZkZDJe0mkfDf5KLdIrZ0dIGS5psne5TTd3yz367x2ZYkNNID3a78tR8TMWSSACVf78EgBVAW/WyL8Yg7/f4+Krz53GGJOeMynVSh/Q4K/Uor18egi3S5ZlF6p8pVf5ztHiwWpIVkFjyIdI7pH/eCROyO/Je0+AkN+TlfMvtuBfU+Hl8++7gp8c7+frL3Sl50x05K+U4uCZYXZuqKYyz5HycliouVvPyBQbawN43C4aKv05F3pZvfzzD9KhgCddOQRWO+diC/4A//7GTbz38nX89yeO8fRbfUDptnYADf5KLdrpgTAdzYtf1buSpts6Z4/8w9EEY5FEetTaXO3P2d9nLBJfdJnnfKpmbeiyWu2cl5uI8D8/fBXVAS9fe/40AM0183dsLWYa/JVahGgiycXxCG11wYXvvArmy/n3zspXN1cHcvb3GV9kR8+FzN7QZWQyVpQjf4DGkJ8/+fmr0l8XU8XSUq39+1elikDPSARjrA6ShcDZIjFX2qdnVr66udrPG93ZXSvHphLLMqGZWe0TT6YIx5JFG/wBbt2xjk/fuT29XqFUafBXahHO2RujtNUVRg7Y73ET9Llzpn2mG5JZgb2pKsBgOEo8mcKbscp3LBLn8sDiOonOpyrgZdwe+Ture2uLOPgDfPK2jrU+hRWnaR+lFqHb3g2rtUBG/mClfobDuUf+IrC+xkn7+DGGrG0kF9vLfyFVAQ+xhLWb12iRtXYoZ3kFfxH5gogcE5E3ROSfRKQ247bPichJEXlLRO7KOH6diByyb/uSlOLOyKrknBuexOsW1i/DAq3lMldzt56RKZpC/vQov7nKrvXPKPc0xlg5/2UYoWdu6FKMrR3KVb4j/yeB3caYq4DjwOcARGQncB+wC7gb+LKIODMnXwEeADrsj7vzPAelVty5oUk21lbgzrMmfjnVBX1zpH0ibKidTk85C70yK37CsSQps7S+PnPJ3NBFg3/xyCv4G2N+ZIxxpvlfApwdKe4FvmmMiRpjTgMngX0isgGoNsa8aKxtcx4BPpDPOSi1GrqHp9I97AtFbdCbe8J3dGpGQ7J11Va54sWMih8nN78caZ/Mzp4a/IvHcub8fwV4wv66BTiXcVu3fazF/nr28ZxE5AER6RSRzv7+/mU8VaWWpnt4smDKPB25Rv7GmKxWxA2VPlwyc0evdEfPZQjS0z39E+kXlWKs8y83CwZ/EXlKRA7n+Lg34z6fBxLA3zuHcjyUmed4TsaYh40xe40xe5uamhY6VaVWxGQswcBErGDKPB11QS9jkTjJ1PRTaHQqzlQ8OaOE0+N20Rjyz8j5O+0YliPtk97NS0f+RWXB93zGmNvnu11EPgr8LPBeM70DcjeQuUNFK9BjH2/NcVypgnXeqfQpsLRPXaUPY6yA77Qhnl3j72iuDszYznEl0j5Ozr/Y2jmXq3yrfe4Gfg94vzFmMuOmx4H7RMQvIluwJnYPGGN6gXERudGu8rkfeCyfc1BqpZ0btv60Wwsw7QMzV/mma/xnLd6yWjxk5PyXMe0zvaFLnJHJ4mvqVq7yfdn/34AfeNKu2HzJGPNrxpgjIvIocBQrHfSgMcZpPP4J4OtABdYcwRNZj6pUAXFq/NvqC2vkn2uVb4+9d+/shmRNVQFePTuS/t5J+yxXnT/AuD3y1+BfHPL6nzfGbJvntoeAh3Ic7wR25/N7lVpN54Ym8XtcNIUKq8lXeuSf0da5d2QKj0toqpp5rs3VfgbDsfQq3+MXx3HJ8uT8/R4XHpekc/61FUvbjF6tDV3hq8pe7+gU8WRqztvPDVllnoW2HjF32sfarnH2egSn1r9/PMrxi+N848A5fvH69kvejjKTiFBlN3cbLdKOnuVIg78qa5F4kvf+6U/4xoGzc96ne2Sy4Cp9AGornbTP9Mj/1ECYlhwT0812rf+FsQh/8NgRqgIePnPXjmU7l1DAau42pmmfoqHBX5W1/vEok7Ekxy6Mz3mfc0NTBVfjD1YffY9L0iP/kckYh7pHuHFrQ9Z919ktHv7mp128eGqQT9+5I10htBxCfq/m/IuMBn9V1gbtxmhO187ZxiJxRqfiBVfmCVa6xervY438nz0xQMrArTuy18Q4q3z/+fUedrdU85F97ct6LlV+DyOTsaJv51xONPirsjZod7o8O0fw7x5yKn0Kb+QP1naOTrXPM8f6qAt6ubq1Nut+DZX+9DzAH71/97L3KAoFPOn1EDUV2im+GOj/kiprgxNW4Dw/PEUimcLjnjkecmr8CzHtA9Yq3+HJGKmU4SfH+3lXR1POwO52CZevr2JPWy3XbVr+DeirAh4u2O0jtJ1zcdDgr8raQNga+SdSht7RSNYI30kHFWLaB6yR/7mhSQ73jDIYjuVM+Tge/+Q7WammpCG/B6fLhKZ9ioOmfVRZc0b+kDvv3z08RcjvSS+oKjTOyP+Zt/oRgVu2zx383S5ZsXLVUMZiMQ3+xUGDvyprQ+FYutY9V96/e3iyIGv8HU5nz2fe6uOqlhoa12ghWpVfg3+x0eCvytrARJQdzVV4XJIz+FsLvAoz3w9W2ieWSPHquRHevWPdmp1HaEbw1xW+xUCDvyprgxMxmqv9tNRVZAV/Y4zVx7/AevpkqrPTUWaOEs/VUpXRJkJH/sVBg78qa4PhKA2Vftrrg1k5/+HJOOFYsmArfcAa+QNzlniuFifnX+F1L0vLCLXy9H9JlS1jDEPhGA0hH231wayR/+mBCQDaC7TGH6ZH/rdsz13iuVqcnL+O+ouHlnqqsjUWSRBPGuorfVRXWCtlxyPxdArjlTNWC+Sr2mrW8jTn1VJXgUvgrl3r1/Q8nJG/Bv/iocFflS1ndW9jyI/XXtx1bmiKnRutANZ5Zoj2+mC6L04haq0L8vzv3caGmrU9x5CO/IuOpn1U2XL6+jSEfOnUjpP6McZw8Mwwe1dgNexy21i79qWozshf2zkXDx35q7LljPwbKv201FoVPc6k75nBSQYmYly3ufCDfyFwNoXRkX/x0JG/KluZI/+aoJfqgCc98u88MwzA3k31a3Z+xcTvceFzu9IT0Krw6chflS2ntYOzI1Z7w3TFz8EzQ1QHPHSsC63Z+RUTEeFLH7mGXRur1/pU1CJp8Fdla3AiSk2FN12X3l4f5FivtalLZ9cw126qw7WG5ZPF5u7da1txpJZG0z6qbA3YNf6Otvog3cNTDIVjnOibKIrJXqUulQZ/VbYGJ6I0ZGxl2F4fJJZM8S+HegG4TvP9qoRp8Fdlaygco6FyugumU+75/149j8cl7Glbu3YJSq00Df6qbA1OzEr72D18Os8Ms2tjNRU+91qdmlIrToO/KkvJlGFoMkZDRv/7jbUV6Z2uNOWjSp0Gf1WWhidjGMOMnL/P42JDjbXYa68u7lIlToO/KktOjX9m2gem8/5a6aNKndb5q7I0GJ5u7ZBpT3st49E466oLt5mbUstBg78qS87Iv3HWyP9379rBp+7YvhanpNSq0uCvypLT1K2+cmbwFxE8bl3Vq0qf5vxVWRoMx3DJ9DaISpUbDf6qLA2GY9RX+tZ060Ol1pIGf1WWrNYO/oXvqFSJ0uCvytLgRCwr369UOdHgr8rS4KyOnkqVGw3+qiwNTERpDGnaR5UvDf6q7MQSKcYjiRmtHZQqNxp6tMEAABFjSURBVBr8VUk6PzLFRx5+idHJeNZtQ/bevfWa9lFlTIO/KkmHukd48dQgR3vHsm4bmMjd2kGpcpJX8BeRPxaRN0TkNRH5kYhszLjtcyJyUkTeEpG7Mo5fJyKH7Nu+JCJaaK2WXSSeAmBkMpZ122A4d2sHpcpJviP/LxhjrjLG7AG+B/wXABHZCdwH7ALuBr4sIs7OGF8BHgA67I+78zwHpbJE4kkAhnOkfZzWDg064avKWF7B3xiT+Z66EjD21/cC3zTGRI0xp4GTwD4R2QBUG2NeNMYY4BHgA/mcg1K5TAf/7JF/OuevE76qjOXd2E1EHgLuB0aB99iHW4CXMu7WbR+L21/PPj7XYz+A9S6B9vb2fE9VlZFIYu60z8BEDK9bqA5oX0NVvhYc+YvIUyJyOMfHvQDGmM8bY9qAvwc+6fxYjocy8xzPyRjzsDFmrzFmb1NT08L/GqVsUTvnnyvtMxyOURf0odNNqpwtOPQxxty+yMf6B+D7wB9gjejbMm5rBXrs4605jiu1rCIJK+0z14SvpnxUucu32qcj49v3A8fsrx8H7hMRv4hswZrYPWCM6QXGReRGu8rnfuCxfM5BqVzmm/AdCke1tYMqe/kmPf+HiOwAUsAZ4NcAjDFHRORR4CiQAB40xiTtn/kE8HWgAnjC/lBqWUXSaZ/cE767a2tW+5SUKih5BX9jzL+Z57aHgIdyHO8Edufze5VaSDSd9slR6hmOaWsHVfZ0ha8qSdGMRV6p1HRNQTxp9fWp19W9qsxp8Fclycn5pwyMRxLp48Pa10cpQIO/KlFOtQ/MzPs7rR007aPKnQZ/VZKctA/MDP7O6t463bhdlTkN/qokRRLJ9Og+c9LXCf5a6qnKnQZ/VZIi8RTN1QEg98hfF3mpcqfBX5WkSDzJhhon+E+P/AfDMUQ07aOUBn9VkqKJFE1VflwyXeED1ure2govbpf29VHlTYO/KkmReJKA101t0JeV9qnTlI9SGvxVaYrGU3bw92ZN+GqZp1Ia/FUJSqYMsWSKgNdFXY6Rv072KqXBX5WgmL2Ri9/jpi7onTHhawV/be2glAZ/VXKc1g4Br4vaoC/d0z+VMgxPxjXtoxQa/FUJclo7BLzOyN8K/qNTcZIpoxO+SqHBX5Ugp5e/M/KPxFNE4kmGJrWvj1IODf6q5Di9/P0ed3pyd3gypqt7lcqQ705eShWczJG/S7wADIfjDE5o8FfKocFflZz0hK/HTdBn/YmPZIz8tambUhr8VQlygr/f6ybkt/7EhyfjDIWjgPb1UQo0569KUDRd5++iLminfSZjDIXjVPrcBLzutTw9pQqCBn9V1N7un5ixRy9k1vlbvX3ASftEdftGpWwa/FXRujAa4Y4/+wlPvnlxxvFoxoSvz+Oi0udmeDLOoK7uVSpNg78qWgMTUVIG+sYiM45nLvIC0p09tambUtM0+KuiFY4mrM+x5Izjzsjf77H+vOsqrc6eQ+GYTvYqZdPgr4pWOGYHf/tFwJGZ8wfSnT2HwjEt81TKpsFfFa2JaNL+PCv4J5K4XYLXbf151wZ99IxMEU2kdIGXUjYN/qpopdM+WSP/FAHP9J92XdDLxTGrxl+Dv1IWDf6qaE0H/5k5/0g8iT+jlr82I8+vE75KWTT4q6LlBH0n9++IJrJH/umvNfgrBWjwV0VsvgnfzFW8dTryVyqLBn9VtJyJ3omstE9qVtpneuSvOX+lLBr8VdGaa8I3mkima/xheuTvc7vSjd6UKnca/FXRmjP4x1MEvNnBv77Sh4is3gkqVcA0+KuiNdeEbyQxK+df6bU/a8pHKYcGf1W0nKAfiadIJFPp45F4koBnOviH/B48LtHJXqUyaPBXRStzZW9mfx9rwnf6T1tEqKv0aWsHpTLo7JcqWuFoApdAylhf11RY6Z1oYubIH+CP791Na13FWpymUgVJg78qWuFoksaQn77xKJMZef/IrAlfgLt3r1/t01OqoGnaRxUlYwzhWIJ11dbmLJm1/rMXeSmlsi1L8BeRT4uIEZHGjGOfE5GTIvKWiNyVcfw6ETlk3/Yl0do7dQmm4kmMgXVVAWC63NMYQzSRmlHnr5TKlvczRETagDuAsxnHdgL3AbuAu4Evi4gzFPsK8ADQYX/cne85qPLjTPY2p0f+1vfpzdt15K/UvJZjePTnwO8Cmbto3wt80xgTNcacBk4C+0RkA1BtjHnRGGOAR4APLMM5qDLj1PjPHvlP79+rwV+p+eQV/EXk/cB5Y8zrs25qAc5lfN9tH2uxv559fK7Hf0BEOkWks7+/P59TVSUmnB7528HfLvWc3r9X0z5KzWfBah8ReQrIVSrxeeA/A3fm+rEcx8w8x3MyxjwMPAywd+/eOe+nyo8T/NdV+Wd872zh6PfoyF+p+SwY/I0xt+c6LiJXAluA1+0521bgFRHZhzWib8u4eyvQYx9vzXFcqSVxVvfWh3y4JCPtk3DSPjryV2o+l/wMMcYcMsasM8ZsNsZsxgrs1xpjLgCPA/eJiF9EtmBN7B4wxvQC4yJyo13lcz/wWP7/DFVunNLOKr+HSp8nPeGb3rxdR/5KzWtFFnkZY46IyKPAUSABPGiMcQqxPwF8HagAnrA/lFoSZ6Rf6fdQ6fdkpH10wlepxVi24G+P/jO/fwh4KMf9OoHdy/V7VXmaGfzd0xO+Ts5f0z5KzUufIaooOaWelT73jJF/OuevaR+l5qXBXxWlcCyB3+PC43ZR6fNkVfvohK9S89NniCpKE9FEekvGSr8nPQE8Hfx15K/UfDT4q6IUjiaotIN/yO+eHvmn2zvon7ZS89FniCpKmcG/0u9Jt3SO6iIvpRZFg78qSuFokpDfCvBW2kcXeSm1FPoMUUUpHEsQ9Nkjf58nvY9vJJ5EBHxu/dNWaj76DFFFaeaEr/UOIBxLpjdv120ilJqfBn9VlKycvxX0nReBcDRhbeSiKR+lFqTPElWUwtFkesI3aH+ejCXSI3+l1Pw0+Kui4+zfG8oo9QSr2VuuzduVUtn0WaKKjrN/b+aEL1hpH928XanF0eCvio5T1plZ6ukcj+jm7Uotij5LVNFJN3VLp30yJnzjSd28XalF0OCvik5mO2eAYGapZyKlaR+lFkGDvyo64XTaJ/fIP6BpH6UWpM8SVXSc/XuDPmuEX+F1p/fxjWjaR6lF0eCvio7TvtkZ8YtIeh/faCKlI3+lFkGfJarozM75g5X311JPpRZPg78qOrmCf6XfY/f20UVeSi2GPktUwZmKJXn2eP+ct2fu3+sI2fv4RhJJ7eWv1CJo8FcF51svn+X+/3OAs4OTOW/P3L/XUenzMDwZxxjt5a/UYuizRBWcYxfG7c9jOW/PbOfsqPR7GApHAd2/V6nF0OCvCs6JvokZn2fL3MLRUel3MzgRA9BST6UWQYO/KijGGI5ftEb+J+zPs+UO/h4mY87+vfpnrdRC9FmiCkrfeJTxiFXNM/fIf3r/XkdmGkjTPkotTIO/KijOqH/XxmpO9k2QTJms+2Tu3+uozPheF3kptTB9lqiCcuKiNdr/mSs3EE2k6B7OrvjJPeE7PdrXkb9SC9PgrwrKib5x6oJebtzaYH1/MTv1k7l/ryNzDkBz/kotTJ8lqqCcuDhBR3MVHc0hAI73ZU/6Zu7f66jUnL9SS6LBXxUMp9KnY12I6oCX9dUBTs4a+c/ev9cR0rSPUkuiwV8VjP7xKGORBNubqwDoaA5ljfxn79/rmDHhqyt8lVqQPktUwThuj/KdlM/25ipO9k2Qyqj4mb1/r2Nmzl9H/kotRIO/KhhOmWfHOnvkvy5EJJ6ie3gqfZ/Z+/c6Zub89c9aqYXos0QVjBN9E9QFvTSGfAB02OmfExmpn1ztnK3vNeev1FJo8FcF48TFcTrWVSEiAGxbZ1f8ZEz6zt6/1xHSUk+llkSfJaogGGM40TeRzvcD1FRYFT8zRv6z9u91VHjdiIDP40q/eCil5uZZ+C5Krbz+8SijU3E61oVmHO9oDs1Y6DV7/16Hs4+vS+O+UouiI39VEJwmbk6Zp6Nj3cyKn7ly/tYxt+b7lVqkvIK/iPyhiJwXkdfsj5/JuO1zInJSRN4Skbsyjl8nIofs274k+h5dMV3ps605e+Q/FU9yfsSq+Jk/+Hs0+Cu1SMsx8v9zY8we++NfAERkJ3AfsAu4G/iyiDjPyq8ADwAd9sfdy3AOqsid6JugNuilKeSfcXy7/WLg5P1z7d/rCPk9Wuap1CKtVM7/XuCbxpgocFpETgL7RKQLqDbGvAggIo8AHwCeWKHz4ON/+zJn5tgLVhWOnpEpdm6szpqs3WbX/H/2O4eoqTjGwEQ0a/9eR9DnxmR3gFZK5bAcwf+TInI/0Al8yhgzDLQAL2Xcp9s+Fre/nn08JxF5AOtdAu3t7Zd0cu31lfi09K/gdTSH+OA1rVnHayq8/OZ7O9Ij/47mELtbanI+xgO3bCUST63oeSpVKhYM/iLyFLA+x02fx0rh/DFg7M9/CvwKkCuPb+Y5npMx5mHgYYC9e/de0pjuv/zczkv5MVVAfvuO7Yu6322XN6/wmShVOhYM/saY2xfzQCLy18D37G+7gbaMm1uBHvt4a47jSimlVlG+1T4bMr79IHDY/vpx4D4R8YvIFqyJ3QPGmF5gXERutKt87gcey+cclFJKLV2+Of//JSJ7sFI3XcCvAhhjjojIo8BRIAE8aIxJ2j/zCeDrQAXWRO+KTfYqpZTKTUyRlEfs3bvXdHZ2rvVpKKVUURGRg8aYvbOPaxmMUkqVIQ3+SilVhjT4K6VUGdLgr5RSZahoJnxFpB84c4k/3ggMLOPpFDu9Htn0msyk1yNbsV6TTcaYptkHiyb450NEOnPNdpcrvR7Z9JrMpNcjW6ldE037KKVUGdLgr5RSZahcgv/Da30CBUavRza9JjPp9chWUtekLHL+SimlZiqXkb9SSqkMGvyVUqoMlXTwF5G77Q3kT4rIZ9f6fNaCiLSJyNMi8qaIHBGR37SP14vIkyJywv5ct9bnuppExC0ir4rI9+zvy/Z6iEitiHxbRI7ZfyfvKOfrASAiv20/Xw6LyDdEJFBq16Rkg7+9YfxfAvcAO4GP2BvLl5sE1vaaVwA3Ag/a1+GzwI+NMR3Aj+3vy8lvAm9mfF/O1+MvgB8YYy4Hrsa6LmV7PUSkBfgNYK8xZjfgBu6jxK5JyQZ/YB9w0hhzyhgTA76JtbF8WTHG9BpjXrG/Hsd6YrdgXYu/te/2t8AH1uYMV5+ItALvA76acbgsr4eIVAO3AF8DMMbEjDEjlOn1yOABKkTEAwSxdhwsqWtSysG/BTiX8f28m8WXAxHZDFwD7Aea7Z3VsD+vW7szW3VfBH4XyNztvVyvx1agH/gbOw32VRGppHyvB8aY88CfAGeBXmDUGPMjSuyalHLwX9Jm8aVORELAd4DfMsaMrfX5rBUR+VmgzxhzcK3PpUB4gGuBrxhjrgHCFHk6I192Lv9eYAuwEagUkX+3tme1/Eo5+M+1iXzZEREvVuD/e2PMd+3DF509mO3PfWt1fqvsZuD9ItKFlQq8TUT+jvK9Ht1AtzFmv/39t7FeDMr1egDcDpw2xvQbY+LAd4GbKLFrUsrB/2WgQ0S2iIgPa8Lm8TU+p1UnIoKVz33TGPNnGTc9DnzU/vqjwGOrfW5rwRjzOWNMqzFmM9bfxL8aY/4d5Xs9LgDnRGSHfei9WHtvl+X1sJ0FbhSRoP38eS/WXFlJXZOSXuErIj+Dld91A//HGPPQGp/SqhORdwLPAYeYznH/Z6y8/6NAO9Yf+88bY4bW5CTXiIjcCnzaGPOzItJAmV4PEdmDNfntA04Bv4w1MCzL6wEgIn8E/CJWtdyrwMeBECV0TUo6+CullMqtlNM+Siml5qDBXymlypAGf6WUKkMa/JVSqgxp8FdKqTKkwV8ppcqQBn+llCpD/x8/vByTWe+WBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
